'''
Copyright 2012 Alessio Rocchi

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
'''

from pyquery import PyQuery
import urllib
import urllib2
import csv
import codecs
import sys
import string
import socket

class Page:
    def __init__(self, url, title):
        self.url = url
        self.title = title
        self.id = None
    
    def fetch(self):
        keepTrying = True
        numTries = 5
        self.clearDom()
        while keepTrying:
            try:
                self.pageDom = PyQuery(url = self.url)
            except:
                numTries -= 1
                if numTries <= 0:
                    keepTrying = False
            else:
                keepTrying = False
            
        
    def clearDom(self):
        if hasattr(self, 'pageDom'):
            del self.pageDom
    
    def buildHref(self, url):
        url = url.replace(' ','%20')
        if url[0] == '/':
            url = self.url[0 : self.url.find('/',7)] + url
        elif url[0:7] == "http://":
            pass
        elif url[0:4] == "www.":
            url = "http://%s" % url
        else:
            url = self.url[0 : self.url.rfind('/') + 1] + url
        return url
    
    def getRedirectedUrl(self, url):
        realUrl = url or ''

        keepTrying = True
        numTries = 3
        if url:
            while keepTrying:
                request = urllib2.Request(url)
                try:
                    response = urllib2.urlopen(request)
                except urllib2.HTTPError, e:
                    realUrl = e.geturl()
                    keepTrying = False
                except:
                    realUrl = ''
                    numTries -= 1
                    if numTries <= 0:
                        keepTrying = False
                else:
                    realUrl = response.geturl()
                    keepTrying = False
                if realUrl != '':
                    if realUrl[0:57] == 'http://www.paginegialle.it/pg/cgi/link_external_site.cgi?':
                        realUrl = realUrl[realUrl.find('http%3A%2F%2F'):].replace('http%3A%2F%2F','http://')

        return realUrl
    
    def buildTitle(self, title):
        return self.title and ( self.title + ' > ' + title ) or title 
        

class categoryPage(Page):
    __catNum = 0
    __excludecategories = []
    def __init__(self, url, title = None, isEntry = False):
        print "Building new category Page @ url %s" % url
        Page.__init__(self, url, title )
        self.__isEntry = isEntry
        self.__subcategories = []
        
        if(not self.isEntry()):
            self.fetch()
        else:
            self.id = self.__class__.__catNum
            self.__class__.__catNum += 1
    
    def jumpSubcategories(self, subcategories, startFromNum):
        for subcategory in subcategories:
            self.__class__.__excludecategories.append(subcategory['Category'])
            self.__class__.__catNum = startFromNum
    
    def getSubcategories(self):
        return self.__subcategories
    
    def getEntryPage(self):
        if self.isEntry():
            return entryPage(self.url, self.title)
        
    def hasSubcategories(self):
        return self.pageDom.find('ul.category-letter li a').size() == 0
    
    def nodeToSub(self, subCategoryNodes):
        return [ {'title' : subCategoryNodes.eq(i).attr.title,
                  'href'  : subCategoryNodes.eq(i).attr.href} for i in range(0,subCategoryNodes.size())]
    
    def fetchCategories(self):
        if self.hasSubcategories():
            subCategoryNodes = self.pageDom.find('div.category-box h3 a')
            subCategoryNodes = self.nodeToSub(subCategoryNodes)
            self.clearDom()
            for subCategoryNode in subCategoryNodes:
                subCategoryEntry = categoryPage(self.buildHref(subCategoryNode['href']), self.buildTitle(subCategoryNode['title']))
                subCategoryEntry.fetchCategories()
                for subCategory in subCategoryEntry.getSubcategories():
                    self.__subcategories.append(subCategory)
        else:
            subCategoryNodes = self.pageDom.find('ul.category-letter li a')
            subCategoryNodes = self.nodeToSub(subCategoryNodes)
            self.clearDom()
            for subCategoryNode in subCategoryNodes:
                if self.buildTitle(subCategoryNode['title']) not in self.__class__.__excludecategories:
                    self.__subcategories.append(categoryPage(self.buildHref(subCategoryNode['href']), self.buildTitle(subCategoryNode['title']), True))
                else:
                    print "~~~Skipping category %s" % self.buildTitle(subCategoryNode['title'])
    
    
    def isEntry(self):
        return self.__isEntry

class entryPage(Page):
    def __init__(self, url, title = None):
        print "Building new entry Page @ url %s" % url
        Page.__init__(self, url, title)
        self.__entries = []
        self.__pageNum = 0
    
    def fetchEntries(self):
        self.fetch()
        entryNodes = self.pageDom.find('div.listing-client-line-pg')
        for i in range(0,entryNodes.size()):
            if entryNodes.eq(i).find('div.client-identifying-pg h3').eq(0).text():
                self.__entries.append({'Name'   : entryNodes.eq(i).find('div.client-identifying-pg h3').eq(0).text().strip().encode('iso-8859-1'),
                                       'Address': (entryNodes.eq(i).find('address p span.locality').eq(0).text() or '').encode('iso-8859-1'),
                                       'Phone'  : (entryNodes.eq(i).find('address p.tel').eq(0).text() or '')[5:].encode('iso-8859-1'),
                                       'City'   : (entryNodes.eq(i).find('address p.street-address').eq(0).text() or '').encode('iso-8859-1'),
                                       'CAP'    : (entryNodes.eq(i).find('address p span.postal-code').eq(0).text() or '').encode('iso-8859-1'),
                                       'Region' : (entryNodes.eq(i).find('address p span.region').eq(0).text() or '')[1:3].encode('iso-8859-1'),
                                       'WebSite': self.getRedirectedUrl(entryNodes.eq(i).find('div.link_prodotti a.lnkwww').eq(0).attr.href)})
                sys.stdout.write(".")
                sys.stdout.flush()
        sys.stdout.write("\n")
    
    def getEntries(self):
        return self.__entries
    
    def isLast(self):
        return self.pageDom.find('a[title=successiva]').size() == 0

    def getPage(self):
        return self.__pageNum
    
    def next(self):
        if self.isLast():
            self.clearDom()
            return False
        else:
            self.__pageNum += 1
            self.__entries = []
            self.url = self.buildHref(self.pageDom.find('a[title=successiva]').eq(0).attr.href)
            return True
        

if __name__ == '__main__':
    socket.setdefaulttimeout(5)
    
    continueDownloading = False
    lastCatNum = 0
    oldCategories = []
    oldEntries = []
    
    try:
        categoriesFile = open('categories.csv', 'r')
        entriesFile = open('entries.csv','r')
        continueDownloading = True
    except:
        continueDownloading = False
    
    if continueDownloading:
        categoriesCsv = csv.DictReader(categoriesFile, ('Category', 'CategoryId'))
        entriesCsv = csv.DictReader(entriesFile, ('Name', 'Phone', 'WebSite', 'Address', 'City', 'CAP', 'Region', 'Category', 'CategoryId'))
        
        try:
            for catRow in categoriesCsv:
                print "Loaded category", catRow
                oldCategories.append(catRow)
                lastCatNum = int(catRow['CategoryId'])
        except:
            pass
        print "Loaded %d categories, press enter to continue" % lastCatNum
        sys.stdin.read(1)
        
        numEntries = 0
        try:
            for entryRow in entriesCsv:
                if int(entryRow['CategoryId']) < lastCatNum:
                    print "Loaded entry %s with CategoryId %d" % (entryRow['Name'], int(entryRow['CategoryId']))
                    oldEntries.append(entryRow)
                    numEntries += 1
        except:
            pass
        print "Loaded %d entries, press enter to continue" % numEntries
        sys.stdin.read(1)
        
        categoriesFile.close()
        entriesFile.close()
    
    categoriesFile = open('categories.csv', 'w', 0)
    entriesFile = open('entries.csv','w', 0)
    categoriesCsv = csv.DictWriter(categoriesFile, ('Category', 'CategoryId'))
    entriesCsv = csv.DictWriter(entriesFile, ('Name', 'Phone', 'WebSite', 'Address', 'City', 'CAP', 'Region', 'Category', 'CategoryId'))
    
    if continueDownloading:
        if oldCategories:
            oldCategories.pop()
        
        print "---------------------------------"
        print "Writing loaded categories to file"
        categoriesCsv.writerows(oldCategories)
        print "Writing loaded entries to file"
        print "---------------------------------\n"
        entriesCsv.writerows(oldEntries)
        
    del oldEntries
    
    pageUrl = {'main'       :'http://www.paginegialle.it/cat/pagine_gialle_naviga.html',
               'test_small' :'http://www.paginegialle.it/cat/informatica_consulenza_e_servizi.html',
               'test_big'   :'http://www.paginegialle.it/cat/informatica_e_telecomunicazioni.html'}['main']
               
    mainCategoryPage = categoryPage(pageUrl)
    mainCategoryPage.jumpSubcategories(oldCategories, lastCatNum)
    del oldCategories
    
    mainCategoryPage.fetchCategories()
    
    for subcategory in mainCategoryPage.getSubcategories():
        
        catInfo = {'Category':subcategory.title, 'CategoryId':subcategory.id}
        categoriesCsv.writerow(catInfo)
        
        entryPag = subcategory.getEntryPage()
        print "::Fetching %s" % entryPag.title
        
        while True:
            entryPag.fetchEntries()
            print "::::page %d, fetched %d" % ( entryPag.getPage(), len(entryPag.getEntries()) )
            for entry in entryPag.getEntries():
                
                entry.update(catInfo)
                
                print "Writing Entry %s to file" % entry['Name']
                entriesCsv.writerow(entry)
                
            if not entryPag.next():
                break
            
    print "Acquisition Complete. GoodBye"
    categoriesFile.close()
    entriesFile.close()
